package com.coderside.fragment.netWork;

import java.io.File;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import org.apache.commons.httpclient.HttpClient;

/**
 * 爬取并分析bling0网站的文章
 * 并存储在数据中，数据库建表脚本在article.sql
 * @author admin
 *
 */
public class ParseBling0 {
	private static String defaultEncode = "UTF-8";

	private static String dirPath = "D:\\bling0";
	private static String baseUrl = "http://www.bling0.com";
	private static Map<String, Pattern> patternMap = new HashMap<String, Pattern>();

	/**
	* 测试
	*
	* @param args
	*/
	public static void main(String[] args) {
		//System.setProperty("http.proxyHost", "165.228.128.10");
		//System.setProperty("http.proxyPort", "3128");
		//System.setProperty("http.proxySet", "true");

		/***当你爬取的内容过于频繁的时候、Iteye会要求输入一个验证码，导致运行异常**/
		String reqUrl = null;
		long a = System.currentTimeMillis();
		for (int i = 1; i <= 2; i++) {
			//			reqUrl = "http://bling0.com/all/latest/page/" + i;
			reqUrl = "http://bling0.com/all/hottest/day/page/" + i;
			parsePage(reqUrl);
		}
		long b = System.currentTimeMillis();
		System.out.println(b - a);
	}

	/**
	 * 保存正则表达式到缓存中并返回
	 * 下次可以直接返回 不用重复编译
	 * PATTERNMODE 为正则表达式的模式参数 e.g:Pattern.CASE_INSENSITIVE
	 * @param regex
	 * @param PATTERNMODE 为-1的时候说明不需要该参数
	 * @return
	 */
	public static Pattern cachePattern(String regex, int PATTERNMODE) {
		Pattern pattern = patternMap.get(regex);
		if (null != pattern) {
			return pattern;
		}
		Pattern p = null;
		if (PATTERNMODE == -1) {
			p = Pattern.compile(regex);
			patternMap.put(regex, p);
		} else {
			p = Pattern.compile(regex, PATTERNMODE);
			patternMap.put(regex, p);
		}
		return p;
	}

	/**
	 * 去掉所有html标签
	 * @param html
	 * @return
	 */
	public static String replaceTag(String html) {
		String regEx = "<.+?>"; //表示标签   
		Pattern p = Pattern.compile(regEx, Pattern.CASE_INSENSITIVE);
		Matcher m = p.matcher(html);
		//		System.out.println(m.find());
		String text = m.replaceAll("");
		text = text.trim();
		text = text.replace("　　", "");
		//		System.out.println(text.replace("　　", ""));
		return text;
	}

	/**
	 * 
	 * @param html
	 */
	public static String parseUrl(String html) {
		String regex = "<[aA]\\s*(href=[^>]+)>(.*?)</[aA]>";
		//在 dotall 模式中，表达式 . 可以匹配任何字符，包括行结束符。默认情况下，此表达式不匹配行结束符。
		Pattern p = Pattern.compile(regex, Pattern.DOTALL);
		Matcher matcher = null;
		matcher = p.matcher(html);
		if (matcher != null && matcher.find()) {
			int a = matcher.groupCount();
			return matcher.group(a);
		}
		return "";
	}

	/**
	 * 每个article有title、score、content、comments、relativetime、group
	 * 使用正则多行匹配article标签内的内容
	 * @param body
	 */
	public static ArticleBling pickupArticle(String articleHtml, ArticleBling article) {
		pickupGroup(articleHtml, article);
		pickupTitle(articleHtml, article);
		pickupScore(articleHtml, article);
		pickupContent(articleHtml, article);
		pickupRelativetime(articleHtml, article);
		return article;
	}

	private static void pickupRelativetime(String articleHtml, ArticleBling article) {
		String regex = "(?i)<time class=\'relativetime\' datetime=\'(.+)\' title=\'(.+)\'>([.+\\s\\S*]+?)</time>";
		Pattern p = cachePattern(regex, Pattern.MULTILINE);//匹配<time class='relativetime' datetime='2012-04-11 20:10:22 +0800' title='2012-04-11 20:10:22 +0800'>约15小时前</time> 提取"2012-04-11 20:10:22"
		Matcher m = p.matcher(articleHtml);//开始编译
		if (m.find()) {
			String relativeTime = m.group(1);
			article.setRelativetime(relativeTime);
		}
	}

	private static void pickupGroup(String articleHtml, ArticleBling article) {
		String regex = "(?i)<a href=\"(.+)\" class=\"group\"(.+)>([.+\\s\\S*]+?)</a>";
		Pattern p = cachePattern(regex, Pattern.MULTILINE);//匹配<a href="/truth" class="group" data-alias="truth">博聆真相调查组</a> 提取"博聆真相调查组"
		Matcher m = p.matcher(articleHtml);//开始编译
		if (m.find()) {
			String groupCode = m.group(1);
			String group = m.group(3);
			article.setGroupCode(groupCode);
			article.setUrl(groupCode + "/" + article.getId());
			article.setGroup(group);
		}
	}

	private static void pickupScore(String articleHtml, ArticleBling article) {

		//		<div class='entry-content'>
		String regex = "(?i)<div class=\'score\'>([.+\\s\\S*]+?)</div>";
		Pattern p = cachePattern(regex, Pattern.MULTILINE);//匹配<div class='entry-content'>开头，</div>结尾的文档
		Matcher m = p.matcher(articleHtml);//开始编译
		if (m.find()) {
			String score = m.group(1);
			article.setScore(score);
		}

	}

	private static String pickupTitle(String article, ArticleBling entry) {
		String regex = "(?i)<h2 class=\'entry-title\'>([.+\\s\\S*]+?)</h2>";
		Pattern p = cachePattern(regex, Pattern.MULTILINE);//匹配<h2 class='entry-title'>开头，</h2>结尾的文档
		Matcher m = p.matcher(article);//开始编译
		if (m.find()) {
			String titleHtml = m.group(1);
			String title = parseUrl(titleHtml);
			entry.setTitle(replaceTag(title));
			return replaceTag(titleHtml);
		}
		return "";
	}

	private static void pickupContent(String articleHtml, ArticleBling article) {
		//		<div class='entry-content'>
		String regex = "(?i)<div class=\'entry-content\'>([.+\\s\\S*]+?)</div>";
		Pattern p = cachePattern(regex, Pattern.MULTILINE);//匹配<div class='entry-content'>开头，</div>结尾的文档
		Matcher m = p.matcher(articleHtml);//开始编译
		if (m.find()) {
			String contentHtml = m.group(1);
			if (hasTag("img", contentHtml)) {
				pickupImage(contentHtml, article);
			}
			String content = replaceTag(contentHtml);
			article.setContent(content);
			regex = "[.+\\s\\S*]+?\\(剩余\\d{1,5}字\\)";
			if (matcherReg(regex, content) && null != article.getUrl()) {
				String articleUrl = baseUrl + article.getUrl();
				System.out.println(articleUrl);
				parseItem(articleUrl, article);
			}

		}
	}

	private static void pickupItemContent(String articleHtml, ArticleBling article) {
		String regex = "(?i)<div class=\'entry-content\'>([.+\\s\\S*]+?)</div>";
		Pattern p = cachePattern(regex, Pattern.MULTILINE);//匹配<div class='entry-content'>开头，</div>结尾的文档
		Matcher m = p.matcher(articleHtml);//开始编译
		if (m.find()) {
			String contentHtml = m.group(1);
			String content = replaceTag(contentHtml);
			article.setContent(content);
		}
	}

	private static void pickupImage(String articleHtml, ArticleBling article) {
		//		String exampleText = "<div class=\"entry-content\"><div class=\"img_link\"><a href=\"/system/pictures/458649/large/%E7%9C%BC%E9%95%9C%E5%BE%88%E9%87%8D%E8%A6%81.jpg?1333322359\" class=\"picture\" id=\"picture-458649\" title=\"事实证明，包装很重要，特别是眼镜.....\"><img alt=\"\" src=\"/system/pictures/458649/thumb/%E7%9C%BC%E9%95%9C%E5%BE%88%E9%87%8D%E8%A6%81.jpg?1333322359\"></a></div><p></p></div>";
		String regex = "<img.+src=\"(.+)\".+?>"; //表示标签   
		Pattern p = cachePattern(regex, Pattern.MULTILINE);//匹配<table .+?>开头，</table>结尾的文档
		Matcher m = p.matcher(articleHtml);
		String localSmallPath = dirPath + "\\" + article.getGroup() + "\\" + article.getId() + "_small.jpg";
		String localLargePath = dirPath + "\\" + article.getGroup() + "\\" + article.getId() + "_large.jpg";
		article.setLocalImg(localSmallPath);
		if (new File(localSmallPath).exists()) {
			return;
		}
		if (m != null && m.find()) {
			String remoteThumbFile = baseUrl + m.group(1);
			String remoteLargeFile = remoteThumbFile.replace("thumb", "large");
			new CopyFile().copyFile(localSmallPath, remoteThumbFile);
			new CopyFile().copyFile(localLargePath, remoteLargeFile);
		}
	}

	public static boolean matcherReg(String regEx, String content) {
		Pattern p = cachePattern(regEx, Pattern.MULTILINE);
		Matcher m = p.matcher(content);
		return m.find();
	}

	/**
	 * 判断是否包含有html标签
	 * @param html
	 * @return
	 */
	public static boolean hasTag(String tag, String html) {
		String regEx = "<" + tag; //表示标签   
		Pattern p = Pattern.compile(regEx, Pattern.CASE_INSENSITIVE);
		Matcher m = p.matcher(html);
		return m.find();
	}

	/**
	 * 每页有25条<article></article>
	 * 使用正则多行匹配table标签内的内容
	 * @param body
	 */
	public static List<ArticleBling> pickupArticlesList(String body) {
		String regex = "(?i)<article\\s.+ id='(.+)'>([.+\\s\\S*]+?)</article>";
		List<ArticleBling> articles = new ArrayList<ArticleBling>();
		Pattern p = cachePattern(regex, Pattern.MULTILINE);//匹配<table .+?>开头，</table>结尾的文档
		Matcher m = p.matcher(body);//开始编译
		while (m.find()) {
			String articleId = m.group(1);
			String id = articleId.split("_")[1];
			ArticleBling article = new ArticleBling();
			article.setId(id);
			pickupArticle(m.group(2), article);
			articles.add(article);
		}
		return articles;
	}

	/**
	 * 解析论坛主页 获取主页上的链接
	 * @param reqUrl
	 */
	public static void parsePage(String pageUrl) {

		//System.setProperty("http.proxyHost", "165.228.128.10");
		//System.setProperty("http.proxyPort", "3128");
		//System.setProperty("http.proxySet", "true");

		//		reqUrl = "http://www.iteye.com/forums/tag/Java";
		Map<String, String> headers = new HashMap<String, String>();
		//		headers.put("Accept-Encoding", "gzip,deflate");

		HttpClient httpClient = HttpClientHelper.getHttpClient();
		String htmlContent = HttpClientHelper.doRequest(httpClient, pageUrl, headers, null, defaultEncode);
		List<ArticleBling> articles = pickupArticlesList(htmlContent);
		DBTool.insertArticles(articles);
	}

	/**
	 * 解析论坛主页 获取主页上的链接
	 * @param reqUrl
	 */
	public static void parseItem(String articleUrl, ArticleBling article) {

		//System.setProperty("http.proxyHost", "165.228.128.10");
		//System.setProperty("http.proxyPort", "3128");
		//System.setProperty("http.proxySet", "true");

		//		reqUrl = "http://www.iteye.com/forums/tag/Java";
		Map<String, String> headers = new HashMap<String, String>();
		//		headers.put("Accept-Encoding", "gzip,deflate");

		HttpClient httpClient = HttpClientHelper.getHttpClient();
		String htmlContent = HttpClientHelper.doRequest(httpClient, articleUrl, headers, null, defaultEncode);
		pickupItemContent(htmlContent, article);
	}

}
